/*=====================================================================
  This file is part of a Microsoft SQL Server Shared Source Application.
  Copyright (C) Microsoft Corporation.  All rights reserved.
 
THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A
PARTICULAR PURPOSE.
======================================================= */

// Purpose:
//    This tool randomly generates a set of MDX queries based on a given template.
//
//	  Input: An XML file.
//           This is the template file. It specifies a collection of information 
//           to be used for query generation (e.g. template query, server, database,
//           cube, token, etc). 
//           The MDX query contains miscellaneous numbers of tokens each denoted by a pair of 
//           vertical bars, e.g. "|city|".
//           The tool replaces tokens with randomly generated Sets. 
//
//    Output: An XML file.
//           It contains a set of queries generated from the template file. 
//
// Aug 2007, new behavior described below.
//
// For the entire user output file, all occurrences of the same template parameter should have the same randomly-selected set.
// The goal is to model a human analyst doing research on a specific set of parameters (e.g. |city|, |week|).
// In the course of work the analyst may execute many MDX queries.  Consider:
// Select ... |city|.parent, |week|.parent ...
// Select ... |city|,        |week|.parent ...
// Select ... |city|.parent, |week| ...
// Select ... |city|,        |week| ...
//
// Originally ASQueryGenerator did  a new random selection of each parameter on a per-query basis.
// Without per-file scope, one cannot model the above scenario.
// With per-file scope, one can model the above scenario, and also model different per-query values 
// by creating more parameters, e.g.|city|, |city2|, etc.
//
// We handle duplicate parameter sets.  
// By duplicate we mean the entire set of parameter values, therefore the entire user output file containing many queries.
// Use <DuplicateRetryCount> to check for uniqueness, and after 'n' retries, to give up.
// After a file is created, we check if it is duplicate, and start over if it is.
// We expect most users will either have duplicates be very rare due to large member sets,
// or quickly get warnings because of trying to generate too many files with small member sets.
//   Typically would be DuplicateRetry=0 (default) to not retry, and DuplicateRetry=100 to give up after many retries.
//   DuplicateRetryCount <=0 : Warn but generate anyway.
//   DuplicateRetryCount > 0 : Warn and retry up to 'n' times, then generate anyway.
// 
// The random number generator is initialized once per entire run.
// This allows for reproducibility of the entire set of files.
// We take some care in initialization to avoid some problems that System.Random has with 
// correlations betwen consecutive seed values.
//
// We support token MDX expressions that depend on other tokens.
// For example:
//   Token Name = time__month
//   Set="NonEmpty( [Time].[Calendar].[Calendar Month].members, {|time_fiscal_week|} * [Measures].[Sale Amt] )"
// This means for every file:
// (1) |time_fiscal_week| member is selected at random from the set of 51 members which was created at the beginning.
// (2) |time_month| is selected at random from the set which is newly created for each file.
// Typically there is only one month for each week, but some might overlap.
// Note that it is implemented using lazy evaluation, so the order does not matter.
// There is no check for cycles in the graph, so a cycle will cause an infinite loop.
//
//--


using System;
using System.IO;
using System.Collections;
using System.Text;
using System.Reflection;
using System.Security.AccessControl;
using System.Xml;
using System.Collections.Generic;
using System.Security.Cryptography;

namespace Microsoft.SqlServer.BISystems.Tool
{
    /// <summary>
    /// This class drives query generation. 
    /// </summary>
    public class ASQueryGenerator
    {
        #region const 

        // We use "|" to indicate a string of token.
        // For example, "select { |city| } on columns from [mycube]".
        public const char TOKEN_SYMBOL = '|';

        // Each line in the client file follows this format: /* file_id : query_id */ query
        // Below are the delimiters we use: 
        private const string OPEN_COMMENT_DELIMITER = "/* ";
        private const string CLOSE_COMMENT_DELIMITER = " */";
        private const string COMMENT_DIVIDER = " : ";

        #endregion const

        #region member variables

        // Name of the template XML file
        private string m_templateFileName;
        
        // Template deserialized from the template XML file.
        private ASQueryTemplate m_template;

        // Checking for DuplicateRetryCount.
        private static Dictionary<string, string> m_dictUniqueFile;
        private static MD5CryptoServiceProvider m_md5CryptoServiceProvider;

        // Uniqueness statistics for each query.
        private static Dictionary<string, bool> [] m_rgDictUnique;

        #endregion member variables

        /// <summary>
        /// Entry point.
        /// </summary>
        /// <param name="args"></param>
        static void Main(string[] args)
        {
            // Verify there is a command line parameter specified.
            // The parameter is the template file name.
            // Otherwise, tell user how to run the tool. 
            if (args.Length != 1)
            {
                Console.WriteLine("Usage: ASQueryGenerator.EXE template_file_name");
                return;
            }

            try
            {
                m_dictUniqueFile = new Dictionary<string, string>();
                m_md5CryptoServiceProvider = new MD5CryptoServiceProvider();

                // Instantiate query generator
                ASQueryGenerator generator = new ASQueryGenerator(args[0]);

                // Start generating...
                generator.Run();
            }
            catch (Exception e)
            {
                // Error handler. 
                while (e != null)
                {
                    Console.WriteLine("================");
                    Console.WriteLine(e.Message);
#if (DEBUG)
                    Console.WriteLine(e.StackTrace);
#endif
                    e = e.InnerException;
                }
            }
        }

        /// <summary>
        /// Default constructor.
        /// </summary>
        /// <param name="fileName">name of the template file</param>
        public ASQueryGenerator(string fileName)
        {
            m_templateFileName = fileName;
            m_template = null;
        }

        /// <summary>
        /// 
        /// </summary>
        public void Run()
        {
            try
            {
                // Does the template file exist?
                if (!File.Exists(m_templateFileName))
                {
                    StringBuilder errMsg = new StringBuilder();
                    errMsg.AppendLine("The specified template file is:");
                    errMsg.AppendLine(m_templateFileName);
                    errMsg.AppendLine("Either the file does not exist or there is no sufficient permission to read the file.");
                    throw new Exception(errMsg.ToString());                    
                }

                // Deserialize the template XML file to ASQueryTemplate object.
                ASQueryTemplateParser parser = new ASQueryTemplateParser();
                m_template = parser.LoadFromXml(m_templateFileName);

                // Retrieve the list of client files that we need to generate. 
                string[] clientFileNames = m_template.ClientFileInfo.GetNames();
                int fileId = m_template.ClientFileInfo.StartingNumber;

                // Create a dictionary for each query.
                // This will hold statistic information we report at the end.
                m_rgDictUnique = new Dictionary<string, bool> [m_template.TemplateQueries.Length];
                for (int iTmp = 0; iTmp < m_rgDictUnique.Length; iTmp++)
                {
                    m_rgDictUnique[iTmp] = new Dictionary<string, bool>();
                }
                string[] rgstrMD5Hash = new string[m_template.TemplateQueries.Length];

                // Retrieve random seed, if it is specified. 
                int seed = 0;
                if (m_template.RandomSeed.HasValue && m_template.RandomSeed.Value != 0)
                    seed = m_template.RandomSeed.Value;

                // Create a random number generator.
                // It is used for the entire set of files.
                // The set of all files is reproducible using the seed.
                System.Random rng = PerfRandom_TransformSeedFunction.CreateSystemRandomTransformSeed(seed);

                foreach (string clientFileName in clientFileNames)
                {
                    int cDuplicateRetry = 0;
                    do
                    {
                        // If the client file already exists, we will overwrite it. 
                        if (File.Exists(clientFileName))
                            File.Delete(clientFileName);

                        XmlWriterSettings settings = new XmlWriterSettings();
                        settings.Indent = true;
                        settings.OmitXmlDeclaration = true;
                        XmlWriter writer = XmlWriter.Create(clientFileName, settings);

                        writer.WriteStartElement("Queries");

                        // For an entire user output file, all occurrences of the same template parameter 
                        // should have the same randomly-selected tuple values.
                        // They are created lazily, and we clear before a new file.
                        m_template.Tokens.ClearSelectedMemberSets();

                        int queryId;
                        for (queryId = 0; queryId < m_template.TemplateQueries.Length; queryId++)
                        {
                            // Generate queries based on the template. 
                            string query = GenerateQuery(m_template.TemplateQueries[queryId], rng);

                            writer.WriteStartElement("Query");
                            writer.WriteString(
                                OPEN_COMMENT_DELIMITER +
                                fileId +
                                COMMENT_DIVIDER +
                                (queryId+1) +
                                CLOSE_COMMENT_DELIMITER +
                                query);
                            writer.WriteEndElement();

                            // Add to per-query statistics.
                            // We want to know for each query template, number of unique instances.
                            // Note that we might have to throw this file away and regenerate.
                            byte[] tmpSource1 = Encoding.Default.GetBytes(query);
                            byte[] hashresult1 = m_md5CryptoServiceProvider.ComputeHash(tmpSource1);
                            string strMD5Hash1 = Convert.ToBase64String(hashresult1);
                            rgstrMD5Hash[queryId] = strMD5Hash1;
                        }

                        // Write out the set of generated values that we used for this file.
                        // This will make it easier to see the differences between two files
                        // when using windiff or similar tools.
                        m_template.Tokens.OutputXMLValues(writer);

                        writer.WriteEndElement();
                        writer.Close();

                        // Check uniqueness of entire file.
                        // Calculate hash of entire file.
                        // We use MD5 hash of parameters to save space.  
                        // The source queries may be hundreds of MB. The parameter set also might be large.
                        // We are OK with the probability of false matches, we think it will be rare.
                        // See http://support.microsoft.com/kb/307020
                        // See http://msdn2.microsoft.com/en-us/library/system.security.cryptography.md5cryptoserviceprovider.aspx
                        string strSetValues2 = m_template.Tokens.OutputXMLValuesAsString();
                        byte[] tmpSource2 = Encoding.Default.GetBytes(strSetValues2);
                        byte[] hashresult2 = m_md5CryptoServiceProvider.ComputeHash(tmpSource2);
                        string strMD5Hash2 = Convert.ToBase64String(hashresult2);

                        // Check if whole file is unique.
                        // key = MD5 hash as a string
                        // value = file name
                        string strOrigClientFileName;
                        if (m_dictUniqueFile.TryGetValue(strMD5Hash2, out strOrigClientFileName))
                        {
                            // Found duplicate.
                            cDuplicateRetry++;
                            Console.WriteLine("Warning: File {0} is a duplicate of {1}, retry count {2}/{3}", 
                                clientFileName, 
                                strOrigClientFileName,
                                cDuplicateRetry,
                                m_template.ClientFileInfo.DuplicateRetryCount);
                            if (cDuplicateRetry <= m_template.ClientFileInfo.DuplicateRetryCount)
                            {
                                // Generate file again.
                                continue;
                            }
                            if (m_template.ClientFileInfo.DuplicateRetryCount <= 0)
                            {
                                // Keep duplicate file.
                            }
                            else
                            {
                                // We exceeded the retry count, so print message and keep file.
                                // Note that we left the last file on disk.
                                Console.WriteLine("Warning: Retry count exceeded maximum {0}.  Keeping duplicate output file.", 
                                    m_template.ClientFileInfo.DuplicateRetryCount);
                            }
                        }
                        else
                        {
                            m_dictUniqueFile.Add(strMD5Hash2, clientFileName);
                        }
                    } while (false);

                    // Update the statistics for each query.
                    // We could not do above because we might have had to regenerate the file.
                    for (int j = 0; j < rgstrMD5Hash.Length; j++)
                    {
                        if (!m_rgDictUnique[j].ContainsKey(rgstrMD5Hash[j]))
                            m_rgDictUnique[j].Add(rgstrMD5Hash[j], true);
                    }

                    // Increment file id. 
                    fileId = fileId + 1;
                }

                // Disconnect from the server.
                TupleDictionary.Disconnect();

                // Output summary statistics for all files.
                StringBuilder strTokenListTotal = new StringBuilder();
                string strTokenList;
                Console.WriteLine("{0,5} {1,6} {2,5} {3,7} {4,7} {5}", 
                    "Query", 
                    "Unique", 
                    "Total", 
                    "Unique%", 
                    "Dup%",
                    "Tokens");
                for (int j = 0; j < m_template.TemplateQueries.Length; j++)
                {
                    strTokenList = GetTokenListAsString(m_template.TemplateQueries[j]);
                    strTokenListTotal.Append(strTokenList);
                    strTokenList = strTokenList.Replace(TOKEN_SYMBOL.ToString(), "");
                    Console.WriteLine("{0,5} {1,6} {2,5} {3,7:0.00%} {4,7:0.00%} {5}",
                        j,
                        m_rgDictUnique[j].Count,
                        clientFileNames.Length,
                        (double)m_rgDictUnique[j].Count / (double)clientFileNames.Length,
                        (double)(clientFileNames.Length - m_rgDictUnique[j].Count) / (double)clientFileNames.Length,
                        strTokenList);
                }
                strTokenList = GetTokenListAsString(strTokenListTotal.ToString());
                strTokenList = strTokenList.Replace(TOKEN_SYMBOL.ToString(), "");
                Console.WriteLine("{0,5} {1,6} {2,5} {3,7:0.00%} {4,7:0.00%} {5}",
                    "Total",
                    m_dictUniqueFile.Count,
                    clientFileNames.Length,
                    (double)m_dictUniqueFile.Count / (double)clientFileNames.Length,
                    (double)(clientFileNames.Length - m_dictUniqueFile.Count) / (double)clientFileNames.Length,
                    strTokenList);
                Console.WriteLine();

                // Output stats on current token sets.
                // Note this will be left over from the very last file.
                // Some tokens are evaluated for each file, so the cardinality may vary.
                TupleDictionary.WriteCardinalityStatistics(this);
            }
            catch (Exception e)
            {
                throw e;
            }
        }

        /// <summary>
        /// Return TRUE if the expression contains token delimiters.
        /// </summary>
        public static bool HasTokenDelimiter(string query)
        {
            // Search for token delimiter ("|").
            int start = query.IndexOf(TOKEN_SYMBOL);
            if (start == -1)
                return false;
            if (start + 1 >= query.Length)
                return false;
            start = query.IndexOf(TOKEN_SYMBOL,start+1);
            if (start == -1)
                return false;
            return true;
        }

        /// <summary>
        /// Return TRUE if the token name has a functional dependency.
        /// </summary>
        public bool IsTokenDependent(string tokenName)
        {
            for (int j=0; j < m_template.Tokens.All.Length; j++)
            {
                if (string.Compare(tokenName,m_template.Tokens.All[j].Name) == 0)
                {
                    return HasTokenDelimiter(m_template.Tokens.All[j].Tuple.Name);
                }
            }
            System.Diagnostics.Debug.Assert(false,"Could not find token");
            return false;
        }

        /// <summary>
        /// Get string containing list of tokens on which source token name depends.
        /// Might be empty string.
        /// </summary>
        public string GetTokenDependencyListAsStringFromTokenName(string tokenName)
        {
            for (int j = 0; j < m_template.Tokens.All.Length; j++)
            {
                if (string.Compare(tokenName, m_template.Tokens.All[j].Name) == 0)
                {
                    return GetTokenListAsString(m_template.Tokens.All[j].Tuple.Name);
                }
            }
            System.Diagnostics.Debug.Assert(false, "Could not find token");
            return "";
        }

        /// <summary>
        /// Return a string containing list of tokens.
        /// Emit each exactly once.
        /// Keep delimiter, to allow us to use this routine on the concatenated output.
        /// </summary>
        public static string GetTokenListAsString(string query)
        {
            Dictionary<string, bool> dict = new Dictionary<string, bool>();
            StringBuilder str = new StringBuilder();

            // Search for token delimiter ("|").
            int iBeg, iEnd = -1;
            for (;;)
            {
                iBeg = query.IndexOf(TOKEN_SYMBOL, iEnd+1);
                if (iBeg == -1)
                    break;
                iEnd = query.IndexOf(TOKEN_SYMBOL, iBeg + 1);
                if (iEnd == -1)
                    break;
                string strToken = query.Substring(iBeg, iEnd - iBeg + 1);
                //string strToken = query.Substring(iBeg + 1, iEnd - iBeg-1); // without delimiter
                if (!dict.ContainsKey(strToken))
                {
                    dict.Add(strToken, true);
                    if (str.Length > 0)
                        str.Append(',');
                    str.Append(strToken);
                }
            }
            return str.ToString();
        }

        /// <summary>
        /// Generate query by replacing the token parameters with their values.
        /// </summary>
        /// <param name="query"></param>
        /// <returns></returns>
        public string GenerateQuery(string query, System.Random rng)
        {
            try
            {
                // Search for token symbol ("|").
                int start = query.IndexOf(TOKEN_SYMBOL);

                // If there is a token symbol ("|") in the query
                while (start != -1)
                {
                    // Find the matching pair 
                    int len = query.Substring(start + 1).IndexOf(TOKEN_SYMBOL);

                    // There is no matching pair. Something wrong with the query. 
                    if (len == -1)
                    {
                        StringBuilder errMsg = new StringBuilder();
                        errMsg.AppendLine("The following query contains one token parameter " + TOKEN_SYMBOL + " with no matching pair.");
                        errMsg.AppendLine(query);
                        throw new Exception(errMsg.ToString());
                    }

                    // Find the name of the token.
                    string tokenName = query.Substring(start, len + 2);

                    // Retrieve token according to its name. 
                    // Note: The token symbol ("|") is present in the query but not in the Token name.
                    //       So we need to remove the token symbol ("|") before we call the indexer.
                    string tokenNameWithoutVerticalBar = tokenName.Substring(1, tokenName.Length - 2);
                    Token tk = m_template.Tokens[tokenNameWithoutVerticalBar];
                    // Check if the token by the given name exists. 
                    if (tk == null)
                    {
                        StringBuilder errMsg = new StringBuilder();
                        errMsg.AppendLine("There is a token in the query. But that token is not defined in the template file.");
                        errMsg.AppendLine("Token: " + tokenNameWithoutVerticalBar);
                        errMsg.AppendLine("Query: " + query);
                        throw new Exception(errMsg.ToString());
                    }

                    // Get token value
                    // For an entire user output file, all occurrences of the same template parameter 
                    // should have the same randomly-selected tuple values.
                    string tokenValue = tk.GenerateSet(this, m_template.Server, m_template.Cube, m_template.Database, rng);

                    // Reconstruct the query by replacing the token
                    query = query.Replace(tokenName, tokenValue);

                    // Get the index of the next "|" (i.e. next token)
                    int offset = start + tokenValue.Length + 1;
                    start = query.Substring(offset).IndexOf(TOKEN_SYMBOL);
                    if (start != -1)
                        start = start + offset;
                }

                return query;
            }
            catch (Exception e)
            {
                throw e;
            }
        }
    }    
}
